{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import brnn_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Model can be \"test\", \"small\", \"medium\", \"large\"\n",
    "model = \"small\"\n",
    "\n",
    "#Put the path to the data here\n",
    "data_path = \"../data\"\n",
    "\n",
    "#Put the path to where you want to save the training data\n",
    "save_path = \"tensorboard/\"\n",
    "\n",
    "# The mixing degree for the prior gaussian mixture\n",
    "# As in Fortunato they report scanning\n",
    "# mix_pi \\in { 1/4, 1/2, 3/4 }\n",
    "mix_pi = 0.25\n",
    "\n",
    "# As in Fortunato they report scanning\n",
    "# log sigma1 \\in { 0, -1, -2 }\n",
    "# log sigma2 \\in { -6, -7, -8 }\n",
    "log_sigma1 = -1.0\n",
    "log_sigma2 = -7.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Starting standard services.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Starting queue runners.\n",
      "INFO:tensorflow:Model/global_step/sec: 0\n",
      "Epoch: 1 Learning rate: 1.000\n",
      "INFO:tensorflow:Recording summary at step 0.\n",
      "0.000 perplexity: 10008.401 speed: 316 wps\n",
      "KL is 126.21958923339844\n",
      "0.004 perplexity: 6226.155 speed: 731 wps\n",
      "KL is 126.21048736572266\n",
      "0.104 perplexity: 924.530 speed: 828 wps\n",
      "KL is 126.01679229736328\n",
      "INFO:tensorflow:Model/global_step/sec: 2.06634\n",
      "INFO:tensorflow:Recording summary at step 248.\n",
      "0.204 perplexity: 686.048 speed: 832 wps\n",
      "KL is 125.74849700927734\n",
      "INFO:tensorflow:Model/global_step/sec: 2.09175\n",
      "INFO:tensorflow:Recording summary at step 499.\n",
      "0.304 perplexity: 560.049 speed: 831 wps\n",
      "KL is 125.54263305664062\n",
      "INFO:tensorflow:Model/global_step/sec: 2.07263\n",
      "INFO:tensorflow:Recording summary at step 748.\n",
      "0.404 perplexity: 486.851 speed: 824 wps\n",
      "KL is 125.3456039428711\n",
      "INFO:tensorflow:Model/global_step/sec: 2.00265\n",
      "INFO:tensorflow:Recording summary at step 988.\n",
      "0.504 perplexity: 437.860 speed: 817 wps\n",
      "KL is 125.13578033447266\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Model/global_step/sec: 1.94037\n",
      "INFO:tensorflow:Recording summary at step 1221.\n",
      "0.604 perplexity: 397.166 speed: 813 wps\n",
      "KL is 124.83613586425781\n",
      "INFO:tensorflow:Model/global_step/sec: 1.95852\n",
      "INFO:tensorflow:Recording summary at step 1456.\n",
      "0.703 perplexity: 367.773 speed: 802 wps\n",
      "KL is 124.60167694091797\n",
      "INFO:tensorflow:Model/global_step/sec: 1.85142\n",
      "INFO:tensorflow:Recording summary at step 1678.\n",
      "0.803 perplexity: 344.232 speed: 797 wps\n",
      "KL is 124.36471557617188\n",
      "INFO:tensorflow:Model/global_step/sec: 1.85743\n",
      "INFO:tensorflow:Recording summary at step 1902.\n",
      "INFO:tensorflow:Model/global_step/sec: 1.61719\n",
      "INFO:tensorflow:Recording summary at step 2095.\n",
      "0.903 perplexity: 323.491 speed: 776 wps\n",
      "KL is 124.08283996582031\n",
      "Epoch: 1 Train Perplexity: 308.345\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 2323.\n",
      "Epoch: 1 Valid Perplexity: 215.985\n",
      "Epoch: 2 Learning rate: 1.000\n",
      "0.000 perplexity: 234.477 speed: 799 wps\n",
      "KL is 123.90294647216797\n",
      "0.004 perplexity: 210.374 speed: 827 wps\n",
      "KL is 123.84458923339844\n",
      "INFO:tensorflow:Recording summary at step 2534.\n",
      "0.104 perplexity: 177.145 speed: 811 wps\n",
      "KL is 123.63219451904297\n",
      "INFO:tensorflow:Recording summary at step 2774.\n",
      "0.204 perplexity: 185.920 speed: 806 wps\n",
      "KL is 123.33939361572266\n",
      "INFO:tensorflow:Recording summary at step 3013.\n",
      "0.304 perplexity: 181.263 speed: 802 wps\n",
      "KL is 123.0980224609375\n",
      "INFO:tensorflow:Recording summary at step 3249.\n",
      "0.404 perplexity: 179.578 speed: 798 wps\n",
      "KL is 122.81614685058594\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 3493.\n",
      "0.504 perplexity: 176.637 speed: 800 wps\n",
      "KL is 122.55200958251953\n",
      "0.604 perplexity: 172.499 speed: 799 wps\n",
      "KL is 122.28764343261719\n",
      "INFO:tensorflow:Recording summary at step 3730.\n",
      "INFO:tensorflow:Recording summary at step 3948.\n",
      "0.703 perplexity: 170.145 speed: 786 wps\n",
      "KL is 122.06147766113281\n",
      "INFO:tensorflow:Recording summary at step 4131.\n",
      "0.803 perplexity: 167.394 speed: 765 wps\n",
      "KL is 121.72598266601562\n",
      "INFO:tensorflow:Recording summary at step 4354.\n",
      "0.903 perplexity: 163.692 speed: 762 wps\n",
      "KL is 121.45817565917969\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 4547.\n",
      "Epoch: 2 Train Perplexity: 161.964\n",
      "Epoch: 2 Valid Perplexity: 161.250\n",
      "Epoch: 3 Learning rate: 1.000\n",
      "0.000 perplexity: 147.809 speed: 809 wps\n",
      "KL is 121.1871566772461\n",
      "0.004 perplexity: 139.975 speed: 811 wps\n",
      "KL is 121.25953674316406\n",
      "INFO:tensorflow:Recording summary at step 4691.\n",
      "INFO:tensorflow:Recording summary at step 4877.\n",
      "0.104 perplexity: 131.004 speed: 640 wps\n",
      "KL is 120.94664001464844\n",
      "INFO:tensorflow:Recording summary at step 5087.\n",
      "0.204 perplexity: 140.754 speed: 669 wps\n",
      "KL is 120.59088134765625\n",
      "INFO:tensorflow:Recording summary at step 5314.\n",
      "0.304 perplexity: 137.358 speed: 697 wps\n",
      "KL is 120.36775970458984\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 5545.\n",
      "0.404 perplexity: 137.547 speed: 713 wps\n",
      "KL is 120.09539794921875\n",
      "INFO:tensorflow:Recording summary at step 5772.\n",
      "0.504 perplexity: 136.381 speed: 721 wps\n",
      "KL is 119.83548736572266\n",
      "INFO:tensorflow:Recording summary at step 5989.\n",
      "0.604 perplexity: 133.984 speed: 718 wps\n",
      "KL is 119.52143096923828\n",
      "INFO:tensorflow:Recording summary at step 6174.\n",
      "0.703 perplexity: 133.592 speed: 698 wps\n",
      "KL is 119.254150390625\n",
      "INFO:tensorflow:Recording summary at step 6351.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 6493.\n",
      "0.803 perplexity: 132.133 speed: 665 wps\n",
      "KL is 119.00086975097656\n",
      "INFO:tensorflow:Recording summary at step 6642.\n",
      "0.903 perplexity: 130.031 speed: 650 wps\n",
      "KL is 118.64723205566406\n",
      "INFO:tensorflow:Recording summary at step 6827.\n",
      "Epoch: 3 Train Perplexity: 129.567\n",
      "Epoch: 3 Valid Perplexity: 143.097\n",
      "Epoch: 4 Learning rate: 1.000\n",
      "0.000 perplexity: 121.542 speed: 579 wps\n",
      "KL is 118.44856262207031\n",
      "0.004 perplexity: 123.657 speed: 481 wps\n",
      "KL is 118.37808227539062\n",
      "INFO:tensorflow:Recording summary at step 6984.\n",
      "INFO:tensorflow:Recording summary at step 7147.\n",
      "0.104 perplexity: 111.076 speed: 536 wps\n",
      "KL is 118.173828125\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 7306.\n",
      "0.204 perplexity: 120.592 speed: 535 wps\n",
      "KL is 117.83321380615234\n",
      "INFO:tensorflow:Recording summary at step 7465.\n",
      "INFO:tensorflow:Recording summary at step 7626.\n",
      "0.304 perplexity: 117.945 speed: 534 wps\n",
      "KL is 117.60139465332031\n",
      "INFO:tensorflow:Recording summary at step 7783.\n",
      "0.404 perplexity: 118.143 speed: 530 wps\n",
      "KL is 117.276611328125\n",
      "INFO:tensorflow:Recording summary at step 7934.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 8088.\n",
      "0.504 perplexity: 117.587 speed: 523 wps\n",
      "KL is 116.99717712402344\n",
      "INFO:tensorflow:Recording summary at step 8249.\n",
      "0.604 perplexity: 115.689 speed: 525 wps\n",
      "KL is 116.71610260009766\n",
      "INFO:tensorflow:Recording summary at step 8398.\n",
      "INFO:tensorflow:Recording summary at step 8530.\n",
      "0.703 perplexity: 116.034 speed: 512 wps\n",
      "KL is 116.42089080810547\n",
      "INFO:tensorflow:Recording summary at step 8668.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 8799.\n",
      "0.803 perplexity: 114.702 speed: 504 wps\n",
      "KL is 116.13665771484375\n",
      "INFO:tensorflow:Recording summary at step 8938.\n",
      "0.903 perplexity: 113.075 speed: 498 wps\n",
      "KL is 115.82427215576172\n",
      "INFO:tensorflow:Recording summary at step 9078.\n",
      "INFO:tensorflow:Recording summary at step 9238.\n",
      "Epoch: 4 Train Perplexity: 113.083\n",
      "Epoch: 4 Valid Perplexity: 140.316\n",
      "Epoch: 5 Learning rate: 0.500\n",
      "0.000 perplexity: 118.734 speed: 630 wps\n",
      "KL is 115.58007049560547\n",
      "0.004 perplexity: 98.236 speed: 639 wps\n",
      "KL is 115.55675506591797\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 9377.\n",
      "INFO:tensorflow:Recording summary at step 9378.\n",
      "INFO:tensorflow:Recording summary at step 9379.\n",
      "INFO:tensorflow:Recording summary at step 9439.\n",
      "0.104 perplexity: 96.266 speed: 201 wps\n",
      "KL is 115.31135559082031\n",
      "INFO:tensorflow:Recording summary at step 9623.\n",
      "0.204 perplexity: 101.293 speed: 297 wps\n",
      "KL is 115.10945129394531\n",
      "INFO:tensorflow:Recording summary at step 9802.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 9984.\n",
      "0.304 perplexity: 98.387 speed: 356 wps\n",
      "KL is 114.90381622314453\n",
      "INFO:tensorflow:Recording summary at step 10167.\n",
      "0.404 perplexity: 97.613 speed: 397 wps\n",
      "KL is 114.64803314208984\n",
      "INFO:tensorflow:Recording summary at step 10347.\n",
      "0.504 perplexity: 96.714 speed: 426 wps\n",
      "KL is 114.53540802001953\n",
      "INFO:tensorflow:Recording summary at step 10532.\n",
      "0.604 perplexity: 94.556 speed: 449 wps\n",
      "KL is 114.29402923583984\n",
      "INFO:tensorflow:Recording summary at step 10714.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 10896.\n",
      "0.703 perplexity: 94.434 speed: 466 wps\n",
      "KL is 114.12970733642578\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Recording summary at step 11081.\n",
      "0.803 perplexity: 92.672 speed: 480 wps\n",
      "KL is 113.99564361572266\n",
      "INFO:tensorflow:Recording summary at step 11262.\n",
      "0.903 perplexity: 90.993 speed: 492 wps\n",
      "KL is 113.81244659423828\n",
      "INFO:tensorflow:Recording summary at step 11449.\n",
      "Epoch: 5 Train Perplexity: 90.609\n",
      "INFO:tensorflow:Recording summary at step 11615.\n",
      "Epoch: 5 Valid Perplexity: 115.375\n",
      "Epoch: 6 Learning rate: 0.250\n",
      "0.000 perplexity: 88.252 speed: 515 wps\n",
      "KL is 113.64664459228516\n",
      "0.004 perplexity: 88.366 speed: 605 wps\n",
      "KL is 113.6902847290039\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 11784.\n",
      "0.104 perplexity: 83.795 speed: 615 wps\n",
      "KL is 113.58868408203125\n",
      "INFO:tensorflow:Recording summary at step 11968.\n",
      "0.204 perplexity: 87.188 speed: 612 wps\n",
      "KL is 113.4096908569336\n",
      "INFO:tensorflow:Recording summary at step 12147.\n",
      "0.304 perplexity: 85.467 speed: 609 wps\n",
      "KL is 113.38579559326172\n",
      "INFO:tensorflow:Recording summary at step 12330.\n",
      "INFO:tensorflow:Recording summary at step 12510.\n",
      "0.404 perplexity: 84.771 speed: 607 wps\n",
      "KL is 113.29771423339844\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 12692.\n",
      "0.504 perplexity: 84.479 speed: 606 wps\n",
      "KL is 113.19418334960938\n",
      "INFO:tensorflow:Recording summary at step 12870.\n",
      "0.604 perplexity: 82.633 speed: 603 wps\n",
      "KL is 113.09888458251953\n",
      "INFO:tensorflow:Recording summary at step 13048.\n",
      "INFO:tensorflow:Recording summary at step 13227.\n",
      "0.703 perplexity: 82.431 speed: 602 wps\n",
      "KL is 113.02318572998047\n",
      "INFO:tensorflow:Recording summary at step 13407.\n",
      "0.803 perplexity: 80.848 speed: 602 wps\n",
      "KL is 112.96913146972656\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 13586.\n",
      "0.903 perplexity: 79.247 speed: 601 wps\n",
      "KL is 112.85610961914062\n",
      "INFO:tensorflow:Recording summary at step 13761.\n",
      "INFO:tensorflow:Recording summary at step 13937.\n",
      "Epoch: 6 Train Perplexity: 78.863\n",
      "Epoch: 6 Valid Perplexity: 106.230\n",
      "Epoch: 7 Learning rate: 0.125\n",
      "0.000 perplexity: 102.363 speed: 591 wps\n",
      "KL is 112.83335876464844\n",
      "0.004 perplexity: 77.216 speed: 553 wps\n",
      "KL is 112.76134490966797\n",
      "INFO:tensorflow:Recording summary at step 14084.\n",
      "0.104 perplexity: 77.743 speed: 587 wps\n",
      "KL is 112.73825073242188\n",
      "INFO:tensorflow:Recording summary at step 14260.\n",
      "0.204 perplexity: 79.833 speed: 593 wps\n",
      "KL is 112.70364379882812\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 14441.\n",
      "INFO:tensorflow:Recording summary at step 14618.\n",
      "0.304 perplexity: 78.939 speed: 592 wps\n",
      "KL is 112.63610076904297\n",
      "INFO:tensorflow:Recording summary at step 14794.\n",
      "0.404 perplexity: 78.421 speed: 592 wps\n",
      "KL is 112.57072448730469\n",
      "INFO:tensorflow:Recording summary at step 14976.\n",
      "0.504 perplexity: 78.163 speed: 593 wps\n",
      "KL is 112.54617309570312\n",
      "INFO:tensorflow:Recording summary at step 15151.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 15335.\n",
      "0.604 perplexity: 76.465 speed: 595 wps\n",
      "KL is 112.5374755859375\n",
      "INFO:tensorflow:Recording summary at step 15515.\n",
      "0.703 perplexity: 76.394 speed: 595 wps\n",
      "KL is 112.48321533203125\n",
      "INFO:tensorflow:Recording summary at step 15693.\n",
      "0.803 perplexity: 74.830 speed: 597 wps\n",
      "KL is 112.44180297851562\n",
      "INFO:tensorflow:Recording summary at step 15875.\n",
      "0.903 perplexity: 73.260 speed: 596 wps\n",
      "KL is 112.45952606201172\n",
      "INFO:tensorflow:Recording summary at step 16053.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 16231.\n",
      "Epoch: 7 Train Perplexity: 72.939\n",
      "Epoch: 7 Valid Perplexity: 102.399\n",
      "Epoch: 8 Learning rate: 0.062\n",
      "0.000 perplexity: 98.339 speed: 622 wps\n",
      "KL is 112.34101867675781\n",
      "0.004 perplexity: 78.393 speed: 498 wps\n",
      "KL is 112.42052459716797\n",
      "INFO:tensorflow:Recording summary at step 16385.\n",
      "0.104 perplexity: 75.128 speed: 594 wps\n",
      "KL is 112.34041595458984\n",
      "INFO:tensorflow:Recording summary at step 16564.\n",
      "0.204 perplexity: 75.713 speed: 598 wps\n",
      "KL is 112.31244659423828\n",
      "INFO:tensorflow:Recording summary at step 16743.\n",
      "INFO:tensorflow:Recording summary at step 16922.\n",
      "0.304 perplexity: 75.430 speed: 597 wps\n",
      "KL is 112.25688171386719\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 17100.\n",
      "0.404 perplexity: 75.346 speed: 596 wps\n",
      "KL is 112.19758605957031\n",
      "INFO:tensorflow:Recording summary at step 17279.\n",
      "0.504 perplexity: 74.807 speed: 597 wps\n",
      "KL is 112.3109359741211\n",
      "INFO:tensorflow:Recording summary at step 17455.\n",
      "INFO:tensorflow:Recording summary at step 17634.\n",
      "0.604 perplexity: 73.289 speed: 595 wps\n",
      "KL is 112.18682861328125\n",
      "INFO:tensorflow:Recording summary at step 17814.\n",
      "0.703 perplexity: 73.195 speed: 595 wps\n",
      "KL is 112.29022216796875\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 17993.\n",
      "0.803 perplexity: 71.785 speed: 595 wps\n",
      "KL is 112.15460968017578\n",
      "INFO:tensorflow:Recording summary at step 18165.\n",
      "INFO:tensorflow:Recording summary at step 18348.\n",
      "0.903 perplexity: 70.197 speed: 595 wps\n",
      "KL is 112.19038391113281\n",
      "INFO:tensorflow:Recording summary at step 18521.\n",
      "Epoch: 8 Train Perplexity: 69.916\n",
      "Epoch: 8 Valid Perplexity: 100.600\n",
      "Epoch: 9 Learning rate: 0.031\n",
      "0.000 perplexity: 77.166 speed: 631 wps\n",
      "KL is 112.14378356933594\n",
      "0.004 perplexity: 72.678 speed: 633 wps\n",
      "KL is 112.1764907836914\n",
      "INFO:tensorflow:Recording summary at step 18677.\n",
      "0.104 perplexity: 73.698 speed: 602 wps\n",
      "KL is 112.1498031616211\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 18854.\n",
      "INFO:tensorflow:Recording summary at step 19031.\n",
      "0.204 perplexity: 73.961 speed: 595 wps\n",
      "KL is 112.15234375\n",
      "INFO:tensorflow:Recording summary at step 19212.\n",
      "0.304 perplexity: 73.722 speed: 595 wps\n",
      "KL is 112.10533905029297\n",
      "INFO:tensorflow:Recording summary at step 19389.\n",
      "0.404 perplexity: 73.949 speed: 595 wps\n",
      "KL is 112.1009292602539\n",
      "INFO:tensorflow:Recording summary at step 19567.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 19745.\n",
      "0.504 perplexity: 73.136 speed: 595 wps\n",
      "KL is 112.10224151611328\n",
      "INFO:tensorflow:Recording summary at step 19925.\n",
      "0.604 perplexity: 71.622 speed: 595 wps\n",
      "KL is 112.06182098388672\n",
      "INFO:tensorflow:Recording summary at step 20103.\n",
      "0.703 perplexity: 71.428 speed: 596 wps\n",
      "KL is 112.10802459716797\n",
      "INFO:tensorflow:Recording summary at step 20284.\n",
      "0.803 perplexity: 70.113 speed: 597 wps\n",
      "KL is 112.04428100585938\n",
      "INFO:tensorflow:Recording summary at step 20466.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 20649.\n",
      "0.903 perplexity: 68.512 speed: 597 wps\n",
      "KL is 112.05294036865234\n",
      "INFO:tensorflow:Recording summary at step 20830.\n",
      "Epoch: 9 Train Perplexity: 68.294\n",
      "Epoch: 9 Valid Perplexity: 99.881\n",
      "Epoch: 10 Learning rate: 0.016\n",
      "0.000 perplexity: 120.003 speed: 630 wps\n",
      "KL is 112.02118682861328\n",
      "0.004 perplexity: 71.833 speed: 649 wps\n",
      "KL is 112.10287475585938\n",
      "INFO:tensorflow:Recording summary at step 20979.\n",
      "0.104 perplexity: 74.002 speed: 598 wps\n",
      "KL is 112.0582046508789\n",
      "INFO:tensorflow:Recording summary at step 21158.\n",
      "INFO:tensorflow:Recording summary at step 21336.\n",
      "0.204 perplexity: 73.665 speed: 597 wps\n",
      "KL is 112.06533813476562\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 21517.\n",
      "0.304 perplexity: 72.941 speed: 597 wps\n",
      "KL is 112.06185150146484\n",
      "INFO:tensorflow:Recording summary at step 21695.\n",
      "0.404 perplexity: 73.250 speed: 596 wps\n",
      "KL is 112.02120208740234\n",
      "INFO:tensorflow:Recording summary at step 21871.\n",
      "INFO:tensorflow:Recording summary at step 22048.\n",
      "0.504 perplexity: 72.097 speed: 594 wps\n",
      "KL is 112.05294036865234\n",
      "INFO:tensorflow:Recording summary at step 22227.\n",
      "0.604 perplexity: 70.561 speed: 595 wps\n",
      "KL is 112.07516479492188\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 22407.\n",
      "0.703 perplexity: 70.608 speed: 595 wps\n",
      "KL is 112.02992248535156\n",
      "INFO:tensorflow:Recording summary at step 22583.\n",
      "INFO:tensorflow:Recording summary at step 22760.\n",
      "0.803 perplexity: 69.222 speed: 594 wps\n",
      "KL is 112.01964569091797\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Recording summary at step 22942.\n",
      "0.903 perplexity: 67.533 speed: 595 wps\n",
      "KL is 112.06886291503906\n",
      "INFO:tensorflow:Recording summary at step 23122.\n",
      "Epoch: 10 Train Perplexity: 67.427\n",
      "Epoch: 10 Valid Perplexity: 99.539\n",
      "Epoch: 11 Learning rate: 0.008\n",
      "0.000 perplexity: 79.856 speed: 634 wps\n",
      "KL is 111.98979949951172\n",
      "0.004 perplexity: 53.624 speed: 644 wps\n",
      "KL is 112.02462005615234\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 23271.\n",
      "INFO:tensorflow:Recording summary at step 23447.\n",
      "0.104 perplexity: 74.561 speed: 586 wps\n",
      "KL is 111.95390319824219\n",
      "INFO:tensorflow:Recording summary at step 23624.\n",
      "0.204 perplexity: 73.886 speed: 587 wps\n",
      "KL is 112.0333023071289\n",
      "INFO:tensorflow:Recording summary at step 23799.\n",
      "0.304 perplexity: 72.406 speed: 589 wps\n",
      "KL is 111.99234008789062\n",
      "INFO:tensorflow:Recording summary at step 23978.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 24151.\n",
      "0.404 perplexity: 73.077 speed: 587 wps\n",
      "KL is 111.94442749023438\n",
      "INFO:tensorflow:Recording summary at step 24330.\n",
      "0.504 perplexity: 71.577 speed: 589 wps\n",
      "KL is 112.05191802978516\n",
      "INFO:tensorflow:Recording summary at step 24507.\n",
      "0.604 perplexity: 70.068 speed: 589 wps\n",
      "KL is 111.98389434814453\n",
      "INFO:tensorflow:Recording summary at step 24685.\n",
      "INFO:tensorflow:Recording summary at step 24862.\n",
      "0.703 perplexity: 70.286 speed: 589 wps\n",
      "KL is 112.00736236572266\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 25039.\n",
      "0.803 perplexity: 68.736 speed: 590 wps\n",
      "KL is 111.93390655517578\n",
      "INFO:tensorflow:Recording summary at step 25221.\n",
      "0.903 perplexity: 67.070 speed: 591 wps\n",
      "KL is 111.99801635742188\n",
      "INFO:tensorflow:Recording summary at step 25401.\n",
      "Epoch: 11 Train Perplexity: 66.895\n",
      "INFO:tensorflow:Recording summary at step 25553.\n",
      "Epoch: 11 Valid Perplexity: 99.317\n",
      "Epoch: 12 Learning rate: 0.004\n",
      "0.000 perplexity: 91.386 speed: 511 wps\n",
      "KL is 111.99177551269531\n",
      "0.004 perplexity: 67.422 speed: 578 wps\n",
      "KL is 111.99104309082031\n",
      "INFO:tensorflow:Recording summary at step 25731.\n",
      "0.104 perplexity: 75.757 speed: 604 wps\n",
      "KL is 111.98400115966797\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 25912.\n",
      "0.204 perplexity: 74.619 speed: 600 wps\n",
      "KL is 111.99166107177734\n",
      "INFO:tensorflow:Recording summary at step 26091.\n",
      "0.304 perplexity: 72.467 speed: 600 wps\n",
      "KL is 111.96537017822266\n",
      "INFO:tensorflow:Recording summary at step 26271.\n",
      "INFO:tensorflow:Recording summary at step 26457.\n",
      "0.404 perplexity: 73.144 speed: 602 wps\n",
      "KL is 112.03225708007812\n",
      "INFO:tensorflow:Recording summary at step 26632.\n",
      "0.504 perplexity: 71.318 speed: 602 wps\n",
      "KL is 111.9737777709961\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 26812.\n",
      "0.604 perplexity: 70.070 speed: 601 wps\n",
      "KL is 112.00979614257812\n",
      "INFO:tensorflow:Recording summary at step 26992.\n",
      "INFO:tensorflow:Recording summary at step 27169.\n",
      "0.703 perplexity: 70.195 speed: 600 wps\n",
      "KL is 111.99178314208984\n",
      "INFO:tensorflow:Recording summary at step 27348.\n",
      "0.803 perplexity: 68.467 speed: 598 wps\n",
      "KL is 111.97626495361328\n",
      "INFO:tensorflow:Recording summary at step 27523.\n",
      "0.903 perplexity: 66.909 speed: 598 wps\n",
      "KL is 112.00092315673828\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 27703.\n",
      "Epoch: 12 Train Perplexity: 66.752\n",
      "INFO:tensorflow:Recording summary at step 27876.\n",
      "Epoch: 12 Valid Perplexity: 99.228\n",
      "Epoch: 13 Learning rate: 0.002\n",
      "0.000 perplexity: 91.343 speed: 600 wps\n",
      "KL is 112.00426483154297\n",
      "0.004 perplexity: 61.256 speed: 494 wps\n",
      "KL is 111.96253967285156\n",
      "INFO:tensorflow:Recording summary at step 28026.\n",
      "0.104 perplexity: 77.067 speed: 588 wps\n",
      "KL is 112.00272369384766\n",
      "INFO:tensorflow:Recording summary at step 28203.\n",
      "0.204 perplexity: 74.531 speed: 591 wps\n",
      "KL is 112.0246353149414\n",
      "INFO:tensorflow:Recording summary at step 28381.\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 28557.\n",
      "0.304 perplexity: 72.911 speed: 588 wps\n",
      "KL is 111.98033142089844\n",
      "INFO:tensorflow:Recording summary at step 28731.\n",
      "0.404 perplexity: 73.322 speed: 588 wps\n",
      "KL is 111.96200561523438\n",
      "INFO:tensorflow:Recording summary at step 28908.\n",
      "0.504 perplexity: 71.177 speed: 589 wps\n",
      "KL is 111.93602752685547\n",
      "INFO:tensorflow:Recording summary at step 29086.\n",
      "INFO:tensorflow:Recording summary at step 29263.\n",
      "0.604 perplexity: 70.301 speed: 589 wps\n",
      "KL is 111.97395324707031\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 29439.\n",
      "0.703 perplexity: 70.324 speed: 589 wps\n",
      "KL is 112.01918029785156\n",
      "INFO:tensorflow:Recording summary at step 29619.\n",
      "0.803 perplexity: 68.492 speed: 589 wps\n",
      "KL is 111.95450592041016\n",
      "INFO:tensorflow:Recording summary at step 29797.\n",
      "INFO:tensorflow:Recording summary at step 29974.\n",
      "0.903 perplexity: 66.987 speed: 590 wps\n",
      "KL is 112.02076721191406\n",
      "INFO:tensorflow:Recording summary at step 30149.\n",
      "Epoch: 13 Train Perplexity: 66.694\n",
      "Epoch: 13 Valid Perplexity: 99.046\n",
      "INFO:tensorflow:Saving checkpoint to path tensorboard/model.ckpt\n",
      "INFO:tensorflow:Recording summary at step 30199.\n",
      "INFO:tensorflow:Recording summary at step 30199.\n",
      "Test Perplexity: 95.833\n",
      "Saving model to tensorboard/.\n"
     ]
    }
   ],
   "source": [
    "brnn_model.main(model_select = model,\n",
    "                   dat_path = data_path,\n",
    "                   sav_path = save_path,\n",
    "                   mixing_pi = mix_pi,\n",
    "                   prior_log_sigma1 = log_sigma1,\n",
    "                   prior_log_sigma2 = log_sigma2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
